package cn._51doit.flink.day10;

import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.CheckpointingMode;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer;

import java.util.Properties;

/**
 * 从Kafka中读取数据，并且将数据保存到Kafka中个，实现ExactlyOnce
 * 为什么可以保证ExactlyOnce呢？
 * 要开启checkpoint，并且将状态持久化同步到statebackend，程序出现异常可以从statebackend恢复
 */
public class KafkaToKafka {

    public static void main(String[] args) throws Exception {

        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        //开启checkpoint同时，并且指定CheckpointingMode为EXACTLY_ONCE
        env.enableCheckpointing(30000, CheckpointingMode.EXACTLY_ONCE);

        //添加KafkaSource（KafkaConsumer）
        //创建KafkaSource
        Properties properties = new Properties();
        properties.setProperty("bootstrap.servers", "node-1.51doit.cn:9092,node-2.51doit.cn:9092,node-3.51doit.cn:9092");
        //没有偏移量从头开始读，有偏移量接着偏移量读
        properties.setProperty("auto.offset.reset", "earliest");
        //不将偏移量写入到Kafka特殊的topic中
        properties.setProperty("enable.auto.commit", "false");
        //调用addSource添加Flink的Connector
        FlinkKafkaConsumer<String> flinkKafkaConsumer = new FlinkKafkaConsumer<String>(
                "topic-in",
                new SimpleStringSchema(),
                properties
        );
        //在checkpoint时不将偏移量写入到Kafka特殊的topic
        //将偏移量写入到OperatorState中
        flinkKafkaConsumer.setCommitOffsetsOnCheckpoints(false);

        DataStreamSource<String> lines = env.addSource(flinkKafkaConsumer);

        //对数据进行处理
        SingleOutputStreamOperator<String> filtered = lines.filter(line -> !line.startsWith("error"));

        //将数据写回到KafkaSink（KafkaProducer）
        FlinkKafkaProducer<String> kafkaProducer = new FlinkKafkaProducer<String>(
               "topic-out",
                new SimpleStringSchema(),
                properties
        );

        filtered.addSink(kafkaProducer);

        env.execute();

    }
}
